//===============================================
//			通用 splider 工具
//===============================================
var http = require('http');
var db = require('../filedb/filedb.js').filedb;

//-抓取一个页面-
function spiderUrl(nPage, opt, fnSpiderData)
{
	var req = http.request(opt, function(res) 
	{
		res.setEncoding('utf8');

		var g_data="";
		res.on('data', function (chunk) 
		{
			g_data+=chunk;
		});

		res.on('end', function()
		{
			console.log("do page " + nPage);
			fnSpiderData(g_data, nPage);		
		});
	});

	req.on('error', function(e) 
	{
		console.log('problem with request ' + opt.path + ' : ' + e.message);
	});

	req.end();
}

//-抓取一批页面-
function spiderAll(options, urls, destobj, fnSpiderData)
{	
	db.open(destobj.path);
	if(destobj.scale)
		db.setDbScale(destobj.scale);

	var destName;
	var opt = options;
	for(var i=0; i<urls.length; i++)
	{
		destName = db.fGet("ul", i, "txt");
		if(!destName)
		{
			opt.path = urls[i];
			spiderUrl(i, opt, fnSpiderData);
		}
	}

	//-不能在此关闭，因为是异步运行-
//	db.save();
//	db.close();
}

//-退出程序保存-
process.on('exit', function () 
{
	console.log('About to exit.');
	db.save();
	db.close();
	console.log('exit.');
});


//===============================================
//			正则表达式数据处理
//===============================================
//-获取所有的信息-
//-- 如果指定nIndex,则只要该匹配项
//-- 否则，将多个指定项作为数组保存	
function reGetSpecMatch(data, re, nIndex)
{
	var rtn  = [];

	var i;
	while ((arr = re.exec(data)) != null)
	{		
		if(arguments.length==3)
		{
			rtn.push(arr[nIndex]);
		}
		else
		{
			var obj = [];
			for(i=2; i<arguments.length; i++)
			{
				obj.push(arr[arguments[i]]);
			}

			rtn.push(obj);
		}				
	}

	return rtn;
}

//-正则匹配，返回匹配数组-
function reMatch(data, re)
{
	var	rtn = data.match(re);
	return rtn;
}

//-正则替换-
function reReplace(data, re, dest)
{
	var rtn = data.replace(re, dest);
	return rtn;
}

//===============================================
//			exports
//===============================================

exports.spiderAll = spiderAll;
exports.reGetSpecMatch = reGetSpecMatch;
exports.reMatch = reMatch;
exports.reReplace = reReplace;
exports.db = db;